{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x10a01d130>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Copyright (c) Microsoft Corporation.\n",
    "# Licensed under the MIT license.\n",
    "\n",
    "import os\n",
    "import re\n",
    "import json\n",
    "import gzip\n",
    "import torch\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "\n",
    "def load_pickle(filename):\n",
    "    with open(filename, \"rb\") as f:\n",
    "        return pickle.load(f)\n",
    "\n",
    "def save_pickle(data, filename):\n",
    "    with open(filename, \"wb\") as f:\n",
    "        pickle.dump(data, f)\n",
    "\n",
    "def load_json(file_path):\n",
    "    with open(file_path, \"r\") as f:\n",
    "        return json.load(f)\n",
    "    \n",
    "def ReadLineFromFile(path):\n",
    "    lines = []\n",
    "    with open(path,'r') as fd:\n",
    "        for line in fd:\n",
    "            lines.append(line.rstrip('\\n'))\n",
    "    return lines\n",
    "\n",
    "def parse(path):\n",
    "    g = gzip.open(path, 'r')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "        \n",
    "'''\n",
    "Set seeds\n",
    "'''\n",
    "seed = 2022\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "short_data_name = \"beauty\" # 'beauty' # \"sports\" # \"clothing\" # \"games\"\n",
    "full_data_name = \"Beauty\" # 'Beauty' # \"Sports_and_Outdoors\" # \"Clothing_Shoes_and_Jewelry\" # \"Video_Games\"\n",
    "if not os.path.exists(os.path.join(\"../../data/\", short_data_name)):\n",
    "    os.mkdir(os.path.join(\"../../data/\", short_data_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# return (user, item, timestamp) sort in get_interaction\n",
    "def Amazon(dataset_name, rating_score=3):\n",
    "    '''\n",
    "    reviewerID - ID of the reviewer, e.g. A2SUAM1J3GNN3B\n",
    "    asin - ID of the product, e.g. 0000013714\n",
    "    reviewerName - name of the reviewer\n",
    "    helpful - helpfulness rating of the review, e.g. 2/3  --\"helpful\": [2, 3],\n",
    "    reviewText - text of the review  --\"reviewText\": \"I bought this for my husband who plays the piano. ...\"\n",
    "    overall - rating of the product  --\"overall\": 5.0,\n",
    "    summary - summary of the review  --\"summary\": \"Heavenly Highway Hymns\",\n",
    "    unixReviewTime - time of the review (unix time)  --\"unixReviewTime\": 1252800000,\n",
    "    reviewTime - time of the review (raw)  --\"reviewTime\": \"09 13, 2009\"\n",
    "    '''\n",
    "    items_with_title = {}\n",
    "    meta_file = f\"../../data/raw_data/meta_{dataset_name}.json.gz\"\n",
    "    with gzip.open(meta_file, \"r\") as fr:\n",
    "        for line in tqdm(fr, desc=\"load meta data\"):\n",
    "            line = eval(line)\n",
    "            if \"title\" not in line:\n",
    "                continue\n",
    "            items_with_title[line['asin']] = 1 \n",
    "\n",
    "    datas = []\n",
    "    data_dict = {}\n",
    "    data_file = f\"../../data/raw_data/reviews_{dataset_name}_5.json.gz\"\n",
    "    \n",
    "    with gzip.open(data_file, \"r\") as fr:\n",
    "        for line in tqdm(fr, desc=\"load all interactions\"):\n",
    "            # try:\n",
    "            line = eval(line)\n",
    "            user = line['reviewerID']\n",
    "            item = line['asin']\n",
    "            if float(line['overall']) <= rating_score or item not in items_with_title: # remove low rating\n",
    "                continue\n",
    "            if (user, item) in data_dict:\n",
    "                continue\n",
    "            time = line['unixReviewTime']\n",
    "            data_dict[(user, item)] = int(time) # merge duplicate interactions, keep the first record\n",
    "            datas.append((user, item, int(time)))\n",
    "    return datas\n",
    "\n",
    "def Amazon_meta(dataset_name, datamaps):\n",
    "    '''\n",
    "    asin - ID of the product, e.g. 0000031852\n",
    "    title - name of the product  --\"title\": \"Girls Ballet Tutu Zebra Hot Pink\",\n",
    "    description\n",
    "    price - price in US dollars (at time of crawl) --\"price\": 3.17,\n",
    "    imUrl - url of the product image (str) --\"imUrl\": \"http://ecx.images-amazon.com/images/I/51fAmVkTbyL._SY300_.jpg\",\n",
    "    related - related products (also bought, also viewed, bought together, buy after viewing)\n",
    "    salesRank - sales rank information --\"salesRank\": {\"Toys & Games\": 211836}\n",
    "    brand - brand name --\"brand\": \"Coxlures\",\n",
    "    categories - list of categories the product belongs to --\"categories\": [[\"Sports & Outdoors\", \"Other Sports\", \"Dance\"]]\n",
    "    '''\n",
    "    meta_datas = {}\n",
    "    meta_file = f\"../../data/raw_data/meta_{dataset_name}.json.gz\"\n",
    "    item_ids = list(datamaps['item2id'].keys())\n",
    "    with gzip.open(meta_file, \"r\") as fr:\n",
    "        for line in tqdm(fr, desc=\"load meta data\"):\n",
    "            line = eval(line)\n",
    "            if line['asin'] not in item_ids:\n",
    "                continue\n",
    "            if \"title\" in line:\n",
    "                line['title'] = re.sub(r'\\n\\t', ' ', line['title']).encode('UTF-8', 'ignore').decode('UTF-8')\n",
    "                line['title'] = line['title'].split(\",\")[0]\n",
    "            if \"description\" in line and type(line['description']) == str:\n",
    "                line['description'] = re.sub(r'\\n\\t', ' ', line['description']).encode('UTF-8', 'ignore').decode('UTF-8')\n",
    "            if 'related' in line:\n",
    "                del line['related']\n",
    "            if 'imUrl' in line:\n",
    "                del line['imUrl']\n",
    "            mapped_id = datamaps['item2id'][line['asin']]\n",
    "            meta_datas[mapped_id] = line\n",
    "    return meta_datas\n",
    "\n",
    "def Amazon_Review(user2id, item2id, dataset_name, rating_score=3):\n",
    "    review_data = {}\n",
    "    data_file = f\"../../data/raw_data/reviews_{dataset_name}_5.json.gz\"\n",
    "    aspect_explanations = None\n",
    "    if os.path.exists(f\"../../data/raw_data/reviews_{dataset_name}.pkl\"):\n",
    "        aspect_explanations = load_pickle(f\"../../data/raw_data/reviews_{dataset_name}.pickle\")\n",
    "    no_sentence = 0\n",
    "\n",
    "    with gzip.open(data_file, \"r\") as fr:\n",
    "        for lidx, line in tqdm(enumerate(fr)):\n",
    "            line = eval(line)\n",
    "            if float(line['overall']) <= rating_score: # remove low rating\n",
    "                continue\n",
    "            user = line['reviewerID']\n",
    "            item = line['asin']\n",
    "            if (user, item) in review_data or user not in user2id or item not in item2id:\n",
    "                continue\n",
    "\n",
    "            if 'reviewText' in line:\n",
    "                exp_ = line\n",
    "                if aspect_explanations is not None:\n",
    "                    exp_ = aspect_explanations[lidx]\n",
    "                    assert exp_['user'] == user and exp_['item'] == item\n",
    "                if 'sentence' in exp_:\n",
    "                    selected_idx = random.randint(0, len(exp_['sentence'])-1)  # randomly sample review of only one feature\n",
    "                    line['explanation'] = exp_['sentence'][selected_idx][2]\n",
    "                    line['feature'] = exp_['sentence'][selected_idx][0]\n",
    "                else:\n",
    "                    no_sentence += 1\n",
    "                line['reviewText'] = re.sub(r'\\n\\t', ' ', line['reviewText']).encode('UTF-8', 'ignore').decode('UTF-8')\n",
    "            review_data[(user, item)] = line\n",
    "\n",
    "    # how to obtain better review data?\n",
    "    print(f\"No sentence: {no_sentence}/{len(review_data)}.\")\n",
    "    return review_data\n",
    "        \n",
    "def add_comma(num): # 1000000 -> 1,000,000\n",
    "    str_num = str(num)\n",
    "    res_num = ''\n",
    "    for i in range(len(str_num)):\n",
    "        res_num += str_num[i]\n",
    "        if (len(str_num)-i-1) % 3 == 0:\n",
    "            res_num += ','\n",
    "    return res_num[:-1]\n",
    "\n",
    "# get user interaction sequence for sequential recommendation\n",
    "def get_interaction(datas):\n",
    "    user_seq = {}\n",
    "    for data in datas:\n",
    "        user, item, time = data\n",
    "        if user in user_seq:\n",
    "            user_seq[user].append((item, time))\n",
    "        else:\n",
    "            user_seq[user] = []\n",
    "            user_seq[user].append((item, time))\n",
    "\n",
    "    for user, item_time in user_seq.items():\n",
    "        item_time.sort(key=lambda x: x[1])  \n",
    "        items = []\n",
    "        for t in item_time:\n",
    "            items.append(t[0])\n",
    "        user_seq[user] = items\n",
    "    return user_seq\n",
    "\n",
    "# K-core user_core item_core, return False if any user/item < core\n",
    "def check_Kcore(user_items, user_core, item_core):\n",
    "    user_count = defaultdict(int)\n",
    "    item_count = defaultdict(int)\n",
    "    for user, items in user_items.items():\n",
    "        for item in items:\n",
    "            user_count[user] += 1\n",
    "            item_count[item] += 1\n",
    "\n",
    "    for user, num in user_count.items():\n",
    "        if num < user_core:\n",
    "            return user_count, item_count, False\n",
    "    for item, num in item_count.items():\n",
    "        if num < item_core:\n",
    "            return user_count, item_count, False\n",
    "    return user_count, item_count, True \n",
    " \n",
    "def filter_Kcore(user_items, user_core, item_core): \n",
    "    user_count, item_count, isKcore = check_Kcore(user_items, user_core, item_core)\n",
    "    while not isKcore:\n",
    "        for user, num in user_count.items():\n",
    "            if user_count[user] < user_core:  \n",
    "                user_items.pop(user)\n",
    "            else:\n",
    "                for item in user_items[user]:\n",
    "                    if item_count[item] < item_core:\n",
    "                        user_items[user].remove(item)\n",
    "        user_count, item_count, isKcore = check_Kcore(user_items, user_core, item_core)\n",
    "    return user_items\n",
    "\n",
    "def id_map(user_items): # user_items dict\n",
    "    user2id = {} # raw 2 uid\n",
    "    item2id = {} # raw 2 iid\n",
    "    id2user = {} # uid 2 raw\n",
    "    id2item = {} # iid 2 raw\n",
    "    user_id = 1  # start from 1\n",
    "    item_id = 1\n",
    "    final_data = {}\n",
    "    random_user_list = list(user_items.keys())\n",
    "    random.shuffle(random_user_list)   \n",
    "    for user in random_user_list:\n",
    "        items = user_items[user]\n",
    "        if user not in user2id:\n",
    "            user2id[user] = str(user_id)\n",
    "            id2user[str(user_id)] = user\n",
    "            user_id += 1\n",
    "        iids = [] # item id lists\n",
    "        for item in items:\n",
    "            if item not in item2id:\n",
    "                item2id[item] = str(item_id)\n",
    "                id2item[str(item_id)] = item\n",
    "                item_id += 1\n",
    "            iids.append(item2id[item])\n",
    "        uid = user2id[user]\n",
    "        final_data[uid] = iids\n",
    "    data_maps = {\n",
    "        'user2id': user2id,\n",
    "        'item2id': item2id,\n",
    "        'id2user': id2user,\n",
    "        'id2item': id2item\n",
    "    }\n",
    "    return final_data, user_id-1, item_id-1, data_maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(data_name, acronym, data_type='Amazon'):\n",
    "    assert data_type in {'Amazon', 'Yelp', 'Steam'}\n",
    "    rating_score = 0.0  # rating score smaller than this score would be deleted\n",
    "    # user 5-core item 5-core\n",
    "    user_core = 5\n",
    "    item_core = 5\n",
    "    attribute_core = 0\n",
    "\n",
    "    datas = Amazon(data_name, rating_score)  # list of [user, item, timestamp]\n",
    "\n",
    "    user_items = get_interaction(datas) # dict of {user: interaction list sorted by time} \n",
    "    print(f'{data_name} Raw data has been processed! Lower than {rating_score} are deleted!')\n",
    "    # raw_id user: [item1, item2, item3...]\n",
    "    user_items = filter_Kcore(user_items, user_core=user_core, item_core=item_core)\n",
    "    print(f'User {user_core}-core complete! Item {item_core}-core complete!')\n",
    "\n",
    "    user_items, user_num, item_num, datamaps = id_map(user_items) # get mapping dicts, randomly shuffle\n",
    "    user_count, item_count, _ = check_Kcore(user_items, user_core=user_core, item_core=item_core)\n",
    "    user_count_list = list(user_count.values()) # user click count\n",
    "    user_avg, user_min, user_max = np.mean(user_count_list), np.min(user_count_list), np.max(user_count_list)\n",
    "    item_count_list = list(item_count.values()) # item click count\n",
    "    item_avg, item_min, item_max = np.mean(item_count_list), np.min(item_count_list), np.max(item_count_list)\n",
    "    interact_num = np.sum([x for x in user_count_list])\n",
    "    sparsity = (1 - interact_num / (user_num * item_num)) * 100\n",
    "    show_info = f'Total User: {user_num}, Avg User: {user_avg:.4f}, Min Len: {user_min}, Max Len: {user_max}\\n' + \\\n",
    "                f'Total Item: {item_num}, Avg Item: {item_avg:.4f}, Min Inter: {item_min}, Max Inter: {item_max}\\n' + \\\n",
    "                f'Iteraction Num: {interact_num}, Sparsity: {sparsity:.2f}%'\n",
    "    print(show_info)\n",
    "\n",
    "\n",
    "    print('Begin extracting meta infos...')\n",
    "    \n",
    "    meta_infos = Amazon_meta(data_name, datamaps)\n",
    "\n",
    "    print(f'{data_name} & {add_comma(user_num)} & {add_comma(item_num)} & {user_avg:.1f}'\n",
    "          f'& {item_avg:.1f} & {add_comma(interact_num)} & {sparsity:.2f}\\% \\\\')\n",
    "\n",
    "    # -------------- Save Data ---------------\n",
    "    data_file = '../../data/{}/'.format(acronym) + 'sequential_data.txt'\n",
    "    metadata_file = '../../data/{}/'.format(acronym) + 'metadata.json'\n",
    "    datamaps_file = '../../data/{}/'.format(acronym) + 'datamaps.json'\n",
    "\n",
    "    with open(data_file, 'w') as out:\n",
    "        for user, items in user_items.items():\n",
    "            out.write(user + ' ' + ' '.join(items) + '\\n')\n",
    "\n",
    "    item_keys = sorted(meta_infos.keys(), key=lambda x: int(x))\n",
    "    print(f\"item2id: {len(datamaps['item2id'])}, meta_infos: {len(meta_infos)}, item_keys: {item_keys[:100]}\")\n",
    "    with open(metadata_file, 'w') as out:\n",
    "        for key in item_keys:\n",
    "            out.write(json.dumps(meta_infos[key]) + '\\n')\n",
    "\n",
    "    json_str = json.dumps(datamaps)\n",
    "    with open(datamaps_file, 'w') as out:\n",
    "        out.write(json_str)\n",
    "\n",
    "    # -------------- Split Train/Valid/Test for Item Import & Tagging ---------------\n",
    "    all_items = [item for item in datamaps['item2id'].keys()]\n",
    "    random.shuffle(all_items)\n",
    "    train_split = int(len(all_items) * 0.8)\n",
    "    valid_split = int(len(all_items) * 0.1)\n",
    "    train_items = all_items[:train_split]\n",
    "    valid_items = all_items[train_split:train_split+valid_split]\n",
    "    test_items = all_items[train_split+valid_split:]\n",
    "    outputs = {'train': train_items, 'val': valid_items, 'test': test_items}\n",
    "    save_pickle(outputs, '../../data/{}/item_splits.pkl'.format(short_data_name))\n",
    "\n",
    "\n",
    "    # -------------- Create Train/Valid/Test for Review ---------------\n",
    "    review_data = Amazon_Review(datamaps['user2id'], datamaps['item2id'], data_name, rating_score)\n",
    "    train_exp_data, valid_exp_data, test_exp_data = [], [], []\n",
    "    train_review_data, valid_review_data, test_review_data = [], [], []\n",
    "    id2user, id2item = datamaps['id2user'], datamaps['id2item']\n",
    "    for user, items in user_items.items():\n",
    "        user = id2user[user]\n",
    "        test_item = id2item[items[-1]]\n",
    "        valid_item = id2item[items[-2]]\n",
    "        test_review_data.append(review_data[(user, test_item)])\n",
    "        if 'explanation' in review_data[(user, test_item)]:\n",
    "            test_exp_data.append(review_data[(user, test_item)])\n",
    "        valid_review_data.append(review_data[(user, valid_item)])\n",
    "        if 'explanation' in review_data[(user, valid_item)]:\n",
    "            valid_exp_data.append(review_data[(user, valid_item)])\n",
    "        for item in items[:-2]:\n",
    "            train_review_data.append(review_data[(user, id2item[item])])\n",
    "            if 'explanation' in review_data[(user, id2item[item])]:\n",
    "                train_exp_data.append(review_data[(user, id2item[item])])\n",
    "    review_outputs = {'train': train_review_data, 'val': valid_review_data, 'test': test_review_data}\n",
    "    save_pickle(review_outputs, '../../data/{}/review_splits.pkl'.format(short_data_name))\n",
    "    exp_outputs = {'train': train_exp_data, 'val': valid_exp_data, 'test': test_exp_data}\n",
    "    save_pickle(exp_outputs, '../../data/{}/exp_splits.pkl'.format(short_data_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '../../data/raw_data/meta_Beauty.json.gz'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb 单元格 5\u001b[0m line \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m main(full_data_name, short_data_name, data_type\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mAmazon\u001b[39;49m\u001b[39m'\u001b[39;49m)\n",
      "\u001b[1;32m/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb 单元格 5\u001b[0m line \u001b[0;36mmain\u001b[0;34m(data_name, acronym, data_type)\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m item_core \u001b[39m=\u001b[39m \u001b[39m5\u001b[39m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m attribute_core \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m datas \u001b[39m=\u001b[39m Amazon(data_name, rating_score)  \u001b[39m# list of [user, item, timestamp]\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m user_items \u001b[39m=\u001b[39m get_interaction(datas) \u001b[39m# dict of {user: interaction list sorted by time} \u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mdata_name\u001b[39m}\u001b[39;00m\u001b[39m Raw data has been processed! Lower than \u001b[39m\u001b[39m{\u001b[39;00mrating_score\u001b[39m}\u001b[39;00m\u001b[39m are deleted!\u001b[39m\u001b[39m'\u001b[39m)\n",
      "\u001b[1;32m/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb 单元格 5\u001b[0m line \u001b[0;36mAmazon\u001b[0;34m(dataset_name, rating_score)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m items_with_title \u001b[39m=\u001b[39m {}\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m meta_file \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m../../data/raw_data/meta_\u001b[39m\u001b[39m{\u001b[39;00mdataset_name\u001b[39m}\u001b[39;00m\u001b[39m.json.gz\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m \u001b[39mwith\u001b[39;00m gzip\u001b[39m.\u001b[39;49mopen(meta_file, \u001b[39m\"\u001b[39;49m\u001b[39mr\u001b[39;49m\u001b[39m\"\u001b[39;49m) \u001b[39mas\u001b[39;00m fr:\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m     \u001b[39mfor\u001b[39;00m line \u001b[39min\u001b[39;00m tqdm(fr, desc\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mload meta data\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/wei_xu/Info_Augmentation/preprocess/step1-Raw_Dataset_Parsing/data_preprocess_amazon.ipynb#W4sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m         line \u001b[39m=\u001b[39m \u001b[39meval\u001b[39m(line)\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.9/gzip.py:58\u001b[0m, in \u001b[0;36mopen\u001b[0;34m(filename, mode, compresslevel, encoding, errors, newline)\u001b[0m\n\u001b[1;32m     56\u001b[0m gz_mode \u001b[39m=\u001b[39m mode\u001b[39m.\u001b[39mreplace(\u001b[39m\"\u001b[39m\u001b[39mt\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     57\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(filename, (\u001b[39mstr\u001b[39m, \u001b[39mbytes\u001b[39m, os\u001b[39m.\u001b[39mPathLike)):\n\u001b[0;32m---> 58\u001b[0m     binary_file \u001b[39m=\u001b[39m GzipFile(filename, gz_mode, compresslevel)\n\u001b[1;32m     59\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mhasattr\u001b[39m(filename, \u001b[39m\"\u001b[39m\u001b[39mread\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mor\u001b[39;00m \u001b[39mhasattr\u001b[39m(filename, \u001b[39m\"\u001b[39m\u001b[39mwrite\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m     60\u001b[0m     binary_file \u001b[39m=\u001b[39m GzipFile(\u001b[39mNone\u001b[39;00m, gz_mode, compresslevel, filename)\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.9/gzip.py:173\u001b[0m, in \u001b[0;36mGzipFile.__init__\u001b[0;34m(self, filename, mode, compresslevel, fileobj, mtime)\u001b[0m\n\u001b[1;32m    171\u001b[0m     mode \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mb\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m    172\u001b[0m \u001b[39mif\u001b[39;00m fileobj \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 173\u001b[0m     fileobj \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmyfileobj \u001b[39m=\u001b[39m builtins\u001b[39m.\u001b[39;49mopen(filename, mode \u001b[39mor\u001b[39;49;00m \u001b[39m'\u001b[39;49m\u001b[39mrb\u001b[39;49m\u001b[39m'\u001b[39;49m)\n\u001b[1;32m    174\u001b[0m \u001b[39mif\u001b[39;00m filename \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    175\u001b[0m     filename \u001b[39m=\u001b[39m \u001b[39mgetattr\u001b[39m(fileobj, \u001b[39m'\u001b[39m\u001b[39mname\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39m'\u001b[39m)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../../data/raw_data/meta_Beauty.json.gz'"
     ]
    }
   ],
   "source": [
    "main(full_data_name, short_data_name, data_type='Amazon')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_test_data(data_name, test_num=99, sample_type='random'):\n",
    "    \"\"\"\n",
    "    sample_type:\n",
    "        random:  sample `test_num` negative items randomly.\n",
    "        pop: sample `test_num` negative items according to item popularity.\n",
    "    \"\"\"\n",
    "\n",
    "    data_file = f'sequential_data.txt'\n",
    "    if sample_type == 'random':\n",
    "        test_file = f'negative_samples.txt'\n",
    "    elif sample_type == 'pop':\n",
    "        test_file = f'negative_samples_pop.txt'\n",
    "\n",
    "    item_count = defaultdict(int)\n",
    "    user_items = defaultdict()\n",
    "\n",
    "    lines = open('../../data/{}/'.format(data_name) + data_file).readlines()\n",
    "    for line in tqdm(lines):\n",
    "        user, items = line.strip().split(' ', 1)\n",
    "        items = items.split(' ')\n",
    "        items = [int(item) for item in items]\n",
    "        user_items[user] = items\n",
    "        for item in items:\n",
    "            item_count[item] += 1\n",
    "\n",
    "    all_item = list(item_count.keys())\n",
    "    count = list(item_count.values())\n",
    "    sum_value = np.sum([x for x in count])\n",
    "    probability = [value / sum_value for value in count]\n",
    "\n",
    "    user_neg_items = defaultdict()\n",
    "\n",
    "    for user, user_seq in tqdm(user_items.items()):\n",
    "        test_samples = []\n",
    "        while len(test_samples) < test_num:\n",
    "            if sample_type == 'random':\n",
    "                sample_ids = np.random.choice(all_item, test_num, replace=False)\n",
    "            else: # sample_type == 'pop':\n",
    "                sample_ids = np.random.choice(all_item, test_num, replace=False, p=probability)\n",
    "            sample_ids = [str(item) for item in sample_ids if item not in user_seq and item not in test_samples]\n",
    "            test_samples.extend(sample_ids)\n",
    "        test_samples = test_samples[:test_num]\n",
    "        user_neg_items[user] = test_samples\n",
    "\n",
    "    with open('../../data/{}/'.format(data_name) + test_file, 'w') as out:\n",
    "        for user, samples in user_neg_items.items():\n",
    "            out.write(user+' '+' '.join(samples)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 35598/35598 [00:00<00:00, 189051.65it/s]\n",
      "100%|██████████| 35598/35598 [02:31<00:00, 235.37it/s]\n"
     ]
    }
   ],
   "source": [
    "sample_test_data(short_data_name, test_num=99, sample_type=\"pop\")  #sample 99 negative testing samples for each user"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "183e66c1b4a3cebcc9a59d658c813a22bf955a401fdefcad593bde0024c573fe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
